#!/usr/bin/env expect
############################################################################
# Purpose: Test of Slurm functionality
#          to be called from test21.30
#          Tests if the MaxJobs limit is enforced
############################################################################
# Copyright (C) 2012 SchedMD LLC
# Written by Nathan Yee <nyee32@schedmd.com>
#
# This file is part of Slurm, a resource management program.
# For details, see <https://slurm.schedmd.com/>.
# Please also read the included file: DISCLAIMER.
#
# Slurm is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# Slurm is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with Slurm; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
############################################################################

# Test MaxJobs limit
proc inc21_30_7 { wait_reason } {
	global salloc scontrol srun acct re_word_str bin_sleep jobmatch job_id1 job_id2 job_id3 number maxjobs_num

	log_info "Starting MaxJob limit test"
	set check_num $maxjobs_num
	array set job_id {}

	# Raise an error to abort the catch block
	set exception_code [catch {

		set jobmatch 0
		for {set inx 0} {$inx <= $check_num} {incr inx} {
			set test_reason 0
			spawn $salloc --account=$acct -t1 -n1 $srun $bin_sleep 60
			expect {
				-re "Granted job allocation ($number)" {
					set job_id($inx) $expect_out(1,string)
					if {$inx < $check_num} {
						log_debug "Job $inx $job_id($inx) was submitted. (inc21.30.7)"
						incr jobmatch
					} else {
						error "Job $inx ($job_id($inx)) wasn't suppose to work but it did, limit was $check_num"
					}
				}
				-re "job ($number) queued and waiting for resources" {
					set job_id($inx) $expect_out(1,string)
					if {$inx >= $check_num} {
						log_debug "Job $inx $job_id($inx) is waiting for resources. This is expected"
						incr jobmatch
					} else {
						set test_reason 1
					}
				}
				-re "error" {
					error "Job allocation should not have failed"
				}
				timeout {
					error "salloc is not responding"
				}
				eof {
					wait
				}
			}

			# Job is waiting, check if the reason is the limit we are testing for
			if {$test_reason != 0 && $job_id($inx) != 0} {
				spawn $scontrol show job $job_id($inx)
				expect {
					-re "Reason=($re_word_str)" {
						if {$expect_out(1,string) ne $wait_reason} {
							error "Job $inx ($job_id($inx)) was suppose to start but it didn't, limit was $check_num"
						} else {
							incr jobmatch
						}
						exp_continue
					}
					timeout {
						error "scontrol is not responding"
					}
					eof {
						wait
					}
				}
			}
		}

		set target_jobs [expr $check_num + 1]
		if {$jobmatch != $target_jobs} {
			error "Not all jobs were not submitted ($jobmatch != $target_jobs)"
		}

		# Checks the job state
		if [info exists job_id($check_num)] {
			subtest [check_job_state $job_id($check_num) PENDING] "Job should be in the PENDING state"
		} else {
			error "Didn't attempt to start enough jobs"
		}

	} message] ; # Store the error message in $message

	# Perform local cleanup
	for {set inx 0} {$inx <= $check_num} {incr inx} {
		if [info exists job_id($inx)] {
			# Cancels the jobs
			cancel_job $job_id($inx)
		}
	}

	# Convert any errors into failures (after cleaning up)
	if {$exception_code == 1} { ; # errors only
		fail "Failure testing MaxJobs limit: $message"
	}
}
